package com.panshi.hujin2.httpcrawler.crawler;

import com.panshi.hujin2.httpcrawler.config.BankEmailInfoConfig;
import com.panshi.hujin2.httpcrawler.config.DownloadPathConfig;
import com.panshi.hujin2.httpcrawler.config.GfwProxyConfig;
import com.panshi.hujin2.httpcrawler.domain.bank.CrawlerBillDoUtil;
import com.panshi.hujin2.httpcrawler.domain.bank.CreditAccountBillDO;
import com.panshi.hujin2.httpcrawler.domain.bank.CreditCardBillDo;
import com.panshi.hujin2.httpcrawler.domain.gmail.GmailAttachmentInfoBO;
import com.panshi.hujin2.httpcrawler.domain.gmail.GmailEmailBO;
import com.panshi.hujin2.httpcrawler.enumtype.BankTypeEnum;
import com.panshi.hujin2.httpcrawler.pdf.*;
import com.panshi.hujin2.httpcrawler.util.gmail.GmailEmailJsonConvertUtil;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.CookieStore;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.stream.Collectors;

/**
 * gmail crawler
 *
 * @Author: wangxl
 * @Date: 2019/1/9 17:05
 */
@Component
public class GmailCrawler {

    private static Logger logger = LoggerFactory.getLogger(GmailCrawler.class);

    /**
     * get bill from gmail
     *
     * @param inputCookie
     * @param bankType
     * @return
     */
    public static List<CreditAccountBillDO> getBillByGmail(String inputCookie, int bankType) throws IOException, URISyntaxException {

        CloseableHttpClient httpClient = null;
        try {
            List<CreditAccountBillDO> resList = new ArrayList<>();
            CookieStore cookieStore = new BasicCookieStore();

            // gfw proxy host and port
            HttpHost gmailProxy = new HttpHost(GfwProxyConfig.staticGfwProxyHost, GfwProxyConfig.staticGfwProxyPort);
            RequestConfig requestConfig = RequestConfig.custom().setRedirectsEnabled(false).build();
            httpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).setDefaultRequestConfig(requestConfig).setProxy(gmailProxy).build();

            String responseContent = getMailSessionData(httpClient, inputCookie);

            // process response content
            List<String> markList = getEmailMark(responseContent);
            String xFrameworkToken = getXFrameworkToken(responseContent);

            // 所有邮件信息
            responseContent = getAllEmailDetail(httpClient, markList, xFrameworkToken, inputCookie);
            // 解析邮件信息
            List<GmailEmailBO> gmailEmailBOList = GmailEmailJsonConvertUtil.convert(responseContent);
            // 循环所有邮件
            if (gmailEmailBOList != null && gmailEmailBOList.size() > 0) {
                for (GmailEmailBO element : gmailEmailBOList) {
                    // 根据邮件发件人地址和邮件标题关键字 过滤非相关邮件
                    if (bankType == BankTypeEnum.SANTANDER.getType()) {
                        if (element.getSenderAddress().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getSantander().get("emailSender"))
                                && element.getEmailTitle().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getSantander().get("titleKeyWord"))) {
                        } else {
                            continue;
                        }
                    } else if (bankType == BankTypeEnum.BRADESCO.getType()) {
                        if (element.getSenderAddress().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getBradesco().get("emailSender"))
                                && element.getEmailTitle().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getBradesco().get("titleKeyWord"))) {
                        } else {
                            continue;
                        }
                    } else if (bankType == BankTypeEnum.CAIXA.getType()) {
                        if (element.getSenderAddress().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getCaixa().get("emailSender"))
                                && element.getEmailTitle().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getCaixa().get("titleKeyWord"))) {
                        } else {
                            continue;
                        }
                    } else if (bankType == BankTypeEnum.ITAU.getType()) {
                        if (element.getSenderAddress().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getItau().get("emailSender"))
                                && element.getEmailTitle().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getItau().get("titleKeyWord"))) {
                        } else {
                            continue;
                        }
                    } else if (bankType == BankTypeEnum.NUBANK.getType()) {
                        if (element.getSenderAddress().equalsIgnoreCase(BankEmailInfoConfig.staticBankEmailInfoMap.getNubank().get("emailSender"))
                                && element.getEmailTitle().contains(BankEmailInfoConfig.staticBankEmailInfoMap.getNubank().get("titleKeyWord"))) {
                        } else {
                            continue;
                        }
                    } else {
                        throw new RuntimeException("no supported bank type");
                    }
                    //如果附件不是pdf文件就跳过
                    List<GmailAttachmentInfoBO> gmailAttachmentInfoBOList = element.getGmailAttachmentInfoBOList();
                    if (gmailAttachmentInfoBOList != null && gmailAttachmentInfoBOList.size() > 0) {

                        GmailAttachmentInfoBO attachmentInfoBO = gmailAttachmentInfoBOList.get(0);
                        if (!attachmentInfoBO.getAttachmentFileName().toLowerCase().endsWith(".pdf")) {
                            break;
                        }
                        String fileName = downloadPdfFile(httpClient, element.getGmailAttachmentInfoBOList().get(0).getAttachmentFileUrl(), inputCookie, DownloadPathConfig.parentPath);
                        //TODO 解析账单,根据银行关键字选择匹配的pdf解析工具
                        CreditCardBillDo creditCardBillDo = BillPdfParseUtil.parsePdf(fileName, bankType);
                        creditCardBillDo.setEmailReceiveDate(element.getReceiveDate());
                        CreditAccountBillDO creditAccountBillDo = CrawlerBillDoUtil.convert(creditCardBillDo);
                        resList.add(creditAccountBillDo);
                    }
                }
            }
            return resList;
        }finally {
            if (httpClient!=null)
                httpClient.close();
        }
    }

    /**
     * get all email unique mark
     *
     * @param httpClient
     * @param inputCookie
     * @return
     * @throws IOException
     * @throws URISyntaxException
     */
    private static String getMailSessionData(CloseableHttpClient httpClient, String inputCookie) throws IOException, URISyntaxException {

        // create resquest
        HttpGet httpGet = new HttpGet();
        httpGet.setURI(new URI("https://mail.google.com/mail/u/0/"));
        httpGet.addHeader("Host", "mail.google.com");
        httpGet.addHeader("Connection", " keep-alive");
        httpGet.addHeader("Accept", " text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        httpGet.addHeader("Upgrade-Insecure-Requests", "1");
        httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        httpGet.addHeader("Referer", " https://myaccount.google.com/");
        httpGet.addHeader("Accept-Encoding", "gzip, deflate, sdch");
        httpGet.addHeader("Accept-Language", "zh-CN,zh;q=0.8");
        httpGet.addHeader("Cookie", inputCookie);
        CloseableHttpResponse httpResponse = httpClient.execute(httpGet);

        HttpEntity responseEntity = httpResponse.getEntity();

        // response content
        return EntityUtils.toString(responseEntity, "UTF-8");
    }

    /**
     * get email unique mark
     *
     * @param responseContent https://mail.google.com/mail/u/0/返回内容
     * @return 所有邮件的标识list
     */
    private static List<String> getEmailMark(String responseContent) {

        int startIndex;
        List<String> list = new ArrayList<>();
        String temp;
        while (true) {
            startIndex = responseContent.indexOf("thread-f");
            if (startIndex == -1) {
                break;
            }
            temp = responseContent.substring(startIndex, startIndex + 40);
            temp = temp.substring(temp.indexOf(":") + 1, temp.indexOf("\\"));
            list.add(temp);
            responseContent = responseContent.replaceFirst("thread-f", "thraed-f");
        }
        if (list.size() > 0) {
            list = list.stream().distinct().collect(Collectors.toList());
        }
        return list;
    }

    /**
     * get X-Framework-Xsrf-Token
     *
     * @param responseContent
     * @return
     */
    private static String getXFrameworkToken(String responseContent) {

        int index = responseContent.indexOf("sdpc");
        if (index == -1) {
            throw new RuntimeException("not found X-Framework-Xsrf-Token");
        }
        String temp = responseContent.substring(index, index + 100);
        temp = temp.replaceAll("\"", "");
        index = temp.indexOf(",");
        temp = temp.substring(index + 1);
        index = temp.indexOf(",");
        String xFrameworkToken = temp.substring(0, index);
        return xFrameworkToken;
    }

    /**
     * get all email info
     *
     * @param httpClient
     * @param list
     * @param xFrameworkToken
     * @param inputCookie
     * @return
     * @throws URISyntaxException
     * @throws IOException
     */
    private static String getAllEmailDetail(CloseableHttpClient httpClient, List<String> list, String xFrameworkToken, String inputCookie) throws URISyntaxException, IOException {

        // create resquest
        HttpPost httpPost = new HttpPost();
        httpPost.setURI(new URI("https://mail.google.com/sync/u/0/i/fd"));
        httpPost.addHeader("Connection", " keep-alive");
        httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        httpPost.addHeader("Origin", "https://mail.google.com");
        httpPost.addHeader("X-Google-BTD", "2");
        httpPost.addHeader("X-Framework-Xsrf-Token", xFrameworkToken);
        httpPost.addHeader("X-Gmail-BTAI", "{\"3\":{\"6\":0,\"10\":1,\"11\":1,\"13\":1,\"15\":0,\"16\":1,\"17\":1,\"18\":0,\"19\":1,\"21\":0,\"22\":1,\"23\":0,\"24\":1,\"25\":1,\"26\":1,\"27\":1,\"28\":1,\"29\":0,\"30\":1,\"31\":1,\"32\":1,\"33\":1,\"34\":1,\"35\":0,\"36\":1,\"37\":\"zh-CN\",\"38\":\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36\",\"39\":1,\"40\":0,\"41\":25,\"42\":0,\"43\":0,\"44\":1,\"45\":0,\"46\":1,\"47\":0,\"48\":1,\"49\":1,\"51\":0},\"5\":\"fd79e3c653\",\"7\":25,\"8\":\"gmail_fe_181225.15_p4\",\"9\":1,\"10\":5,\"11\":\"\",\"12\":28800000,\"13\":\"+08:00\",\"14\":1,\"16\":227864439,\"17\":\"\",\"18\":\"\",\"19\":\"" +
                String.valueOf(System.currentTimeMillis()) + "\"}");
        httpPost.addHeader("Content-Type", "application/json");
        httpPost.addHeader("Accept", "*/*");
        httpPost.addHeader("Referer", "https://mail.google.com/mail/u/0/");
        httpPost.addHeader("Accept-Encoding", "gzip, deflate");
        httpPost.addHeader("Accept-Language", "zh-CN,zh;q=0.8");
        httpPost.addHeader("Cookie", inputCookie);

        String body = getRequestBody(list);
        StringEntity httpEntity = new StringEntity(body);
        httpPost.setEntity(httpEntity);

        CloseableHttpResponse httpResponse = httpClient.execute(httpPost);

        HttpEntity responseEntity = httpResponse.getEntity();

        // response content
        return EntityUtils.toString(responseEntity, "UTF-8");
    }

    /**
     * request to download pdf file
     *
     * @param httpClient
     * @param url
     * @return
     * @throws URISyntaxException
     * @throws IOException
     */
    private static String downloadPdfFile(CloseableHttpClient httpClient, String url, String inputCookie, String parentPath) throws URISyntaxException, IOException {

        // create resquest
        HttpGet httpGet = new HttpGet();
        httpGet.setURI(new URI(url));
        httpGet.addHeader("Host", "mail.google.com");
        httpGet.addHeader("Connection", "keep-alive");
        httpGet.addHeader("Upgrade-Insecure-Requests", "1");
        httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        httpGet.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        httpGet.addHeader("Referer", "https://mail.google.com/mail/u/0/?pli=1");
        httpGet.addHeader("Accept-Encoding", "gzip, deflate, br");
        httpGet.addHeader("Accept-Language", "zh-CN,zh;q=0.9");
        httpGet.addHeader("cache-control", "no-cache");
        httpGet.addHeader("Cookie", inputCookie);

        CloseableHttpResponse httpResponse = httpClient.execute(httpGet);

        // get location url
        Header[] headers = httpResponse.getAllHeaders();
        String location = null;
        for (Header h : headers) {
            if (h.getName().equals("Location")) {
                location = h.getValue();
            }
        }
        if (location == null) {
            throw new RuntimeException("redirect download error");
        }

        // redirection request
        httpGet = new HttpGet();
        httpGet.setURI(new URI(location));
        httpGet.addHeader("Host", "mail-attachment.googleusercontent.com");
        httpGet.addHeader("Connection", "keep-alive");
        httpGet.addHeader("Upgrade-Insecure-Requests", "1");
        httpGet.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36");
        httpGet.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
        httpGet.addHeader("Referer", "https://mail.google.com/mail/u/0/?ui=2&ik=ass9e3c653&attid=0.1&permmsgid=msg-f:1622143800419840319&th=033126a0713f&view=att&disp=safe");
        httpGet.addHeader("Accept-Encoding", "gzip, deflate, br");
        httpGet.addHeader("Accept-Language", "zh-CN,zh;q=0.9");
        httpGet.addHeader("cache-control", "no-cache");

        File file = null;
        InputStream is = null;
        FileOutputStream fos = null;
        BufferedInputStream bis = null;
        BufferedOutputStream bos = null;
        try {
            httpResponse = httpClient.execute(httpGet);
            // content-type
            headers = httpResponse.getAllHeaders();
            String contentType = null;
            for (Header h : headers) {
                if (h.getName().equals("Content-Type")) {
                    contentType = h.getValue();
                }
            }
            if (contentType == null || !contentType.toLowerCase().contains("pdf")) {
                throw new RuntimeException("redirect download error");
            }
            // download pdf file
            is = httpResponse.getEntity().getContent();
            file = new File(parentPath + System.currentTimeMillis() + String.valueOf(new Random().nextInt(1000)) + ".pdf");
            fos = new FileOutputStream(file);
            bis = new BufferedInputStream(is);
            bos = new BufferedOutputStream(fos);
            int len = -1;
            byte[] b = new byte[1024];
            while ((len = bis.read(b)) != -1) {
                bos.write(b, 0, len);
            }

        } finally {
            if (bis != null) {
                bis.close();
            }
            if (bos != null) {
                bos.close();
            }
            if (is != null) {
                is.close();
            }
            if (fos != null) {
                fos.close();
            }
        }
        return file.getAbsolutePath();
    }

    /**
     * 转化请求体
     *
     * @param list
     * @return
     */
    private static String getRequestBody(List<String> list) {
        if (list == null || list.size() == 0) {
            return null;
        }
        StringBuilder sb = new StringBuilder("");
        sb.append("{\"1\":[");

        for (int i = 0; i < list.size(); i++) {
            sb.append("{\"1\":\"thread-f:");
            sb.append(list.get(i));
            sb.append("\",\"2\":1,\"5\":1}");
            if (i < list.size() - 1) {
                sb.append(",");
            }
        }
        sb.append("],\"2\":2}");
        return sb.toString();
    }

    public static void main(String[] args) throws IOException, URISyntaxException {
        String cookie = "GMAIL_AT=AF6bupPXvTQ2sjwhYsRDJQljiH_EPpCCGw; COMPASS=gmail=CpcBAAlriVfS9rglTvf-doIiDeChl2fj_le4teFFqTawqloo1YsvME2fynnvA6wwOH-XC9lBaon_7gDRUIPCPRu6orBBdH8FDSuAXQTEBFOTobu2OZ5Z4vcrLsCVhOcIK7DNXoI0cRBgy04Cc4RV8g18Kanq715qVNypE1XB2IsEFZtOI3s_yZBcVF4C1MMqK5lslUbE1IN30BDsvfvhBRqmAQAJa4lXBX5LJaTzyzyqQHc_qDcKLn1TfjCvkbqyplEbid8SsV_jWE5L3byW5bRaMiLgUnKNYVpfp8TK7n4upr_KlQk4q69nduK8HOxqMGFlQ9ABck44g4lqxBV0diTONK-9fYVYad08tKVCWv_5a1AgY8syD7fhDQc6S96wVxhFqe_Nedk06y6FQNtufbAdztFWNyATfExF7ma9FgGEZWn2DA4eS6s; GMAIL_IMP=v*2%2Fad-tr-pr*2%2Fad-tr-so*2%2Ftl-ard-s*485%2Ftl-tbsfd*600924%2Fsdim*185%2Fsact*3%2Ftl-ar-s*1%2Ftl-ar-f*1; COMPASS=gmail=CpcBAAlriVfBoCblhykM6U8f-cvsUa7NkgYiFK10sB2VZ36YpqWc-9G_lNrXDHEvpDj4GqSAj5Rsq0LxXAIIywXSBBxTz8n8kBG3hXsyoluqFw-nyWns-OY2dtg5j_2ksMqSEb7Ojhxd7cIGcRpUTPp0GEhwmdKSr7sWBqT4AyFLtkkzt9BhlCrA22Vqj1uE7aKVYqa1w3JkyxCR8tbhBRqmAQAJa4lX4kp5xyO_vjl530sWxh14djW3JUNVXIvFz_ZzQa-8lin3YTYOQ33zUo6eh2QgkVy02bG2zIuxrhw9WbUTdeuWn-5BEFEglRiWHXabsnPprqZwXysHeW_L7ap2xcItPSVF72XijCjuSRZkSbUhtcUIBppGuIQqIo6poNrTDtQEIx4mdXpv6R8PosBLKefdAOwo8v8U-RnHAq0RupUnghTZU2c; HSID=AXcYnFWkXYysESK31; SSID=AoTs1mB8W76Zo-xPl; APISID=y84XNrD6XwHARjzj/AxMFE9v2YbkHXmYov; SAPISID=NYV67d6KZmOgnIn_/ABEUaQg2Uu0usmQnw; CONSENT=YES+JP.zh-CN+20171003-07-0; __guid=29003808.1629440147437527600.1547082961038.2935; SID=8AYnfcMJfYQJqhwjgj4M5l-spvVpvY6EJ91Ip98baACfc1shY9lyq5llaeWMFRDTBNFs3w.; monitor_count=8; 1P_JAR=2019-01-16-03; NID=154=Do_dwLzNzQVAaIvx5JucPnJRKb1btasmAC2NdH6awHgTFX0OB5P777srosQCW9pmeeXNcuZqAfDSW-rMXLRcks1PA5jPZw_0HooFx45s-nadvFXJL_3e1M6Pif1oTUBrkzGDgGcOM0dWQh_AmRY4SMGxa_sfeDQS2JPHuIQ5RWmyPi59BGdGAc0I; OSID=9QYnfVX03v50qxN-QfSH7Jw6E5lbf3k7eoecEoUCYt0cZyuFn1GB9oG2smrECVp_Xyv_Pg.; SIDCC=ABtHo-HFo-WSVFG5j6M9a2jYmWb-jDg9r6s_DuXwfeQSMVrxnC7Gpp92ijdMcKvJeZ-694cURg";
        getBillByGmail(cookie, 1);

    }

}
